# 爬取电商任一商品的评论数据
# 电商网站任选其一：京东、淘宝、拼多多，电商APP任选其一：京东、淘宝、得物、抖音、拼多多
# 分析页面，结合正则化，爬取评论数据，保存为csv文件，字段不限，数据量不少于1w条，推荐requests、selenium，scrapy

# 步骤如下:
# 目标爬取京东的某一商品评论数据---商品链接: https://item.jd.com/59202070807.html
# 1.写好爬取解析代码
# 2.利用线程池进行高效爬取
# 3.导入ProblemA.csv文件

import requests
from concurrent.futures import ThreadPoolExecutor
import re
import csv
import time

# 读取Cookie
with open("Cookie.txt", mode="r") as c:
    Cookie = c.read()

# 写请求头
Header = {
    "Cookie": Cookie,
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"
}

url = "https://item.jd.com/59202070807.html"
api1 = "https://api.m.jd.com/?appid=item-v3&functionId=pc_club_productPageComments&client=pc&clientVersion=1.0.0&t" \
       "=1712287076781&loginType=3&uuid=181111935.17122858726521436258891.1712285873.1712285873.1712285873.1&productId" \
       "=59202070807&score=0&sortType=5&page="
api2 = "&pageSize=10&isShadowSku=0&fold=1&bbtf=&shield= "


def get_goods_name():
    resp = requests.get(url, headers=Header)

    # 获取标题作为商品名

    goods = re.search(r"<title>(?P<name>.*?)</title>", resp.text).group("name")

    return goods.split(" ")[1]


def get_goods_comment(page):
    resp = requests.get(api1 + f"{page}" + api2, headers=Header)
    for j in range(10):
        user_name = resp.json()["comments"][j]['nickname']
        user_comment = resp.json()["comments"][j]['content']
        create_time = resp.json()["comments"][j]['creationTime']
        csvwriter.writerow([user_name, user_comment, create_time])

    # 防止被封
    time.sleep(1)


if __name__ == '__main__':
    # 初始化评论文件
    f = open("bak/comments.csv", mode="w", encoding="utf-8")
    csvwriter = csv.writer(f)
    csvwriter.writerow(["评论抓取", "商品名称", get_goods_name()])
    csvwriter.writerow(["用户名", "用户评论", "评论时间"])

    # 创建线程池
    with ThreadPoolExecutor(100) as t:
        for i in range(1000):
            t.submit(get_goods_comment, page=i)

    # get_goods_comment(0)
    f.close()
    print("over!")
